package com.webull.information.center.carwler.common.util.news;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Optional;

import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.aop.scope.ScopedProxyUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.fasterxml.jackson.databind.JsonNode;
import com.webull.framework.util.UtilJson;
import com.webull.information.center.carwler.common.util.jsoup.HttpJsoupClient;
import com.webull.information.center.common.constants.Constants;

/**
 * 新浪个股新闻资讯列表解析工具
 *
 * @author shimingjun
 * @version 1.0
 * @date 2016年8月26日 下午3:51:37
 * @since JDK 1.8
 */
@Component
public class YahooNewsListUitl {
    private @Autowired
    HttpJsoupClient htmlPrase;
    protected final Logger logger = LogManager.getLogger(getClass());

    /**
     * yahoo香港网站的 繁体新闻 https://hk.finance.yahoo.com/q?s=AAPL
     *
     * @param url
     * @return
     */
    public List<YahooItem> parseZhantNewsList(String url) {
        Document doc = htmlPrase.parseDocumentFromUrl(url, true);
        if (doc == null) {
            logger.warn("SinaNewsListUitl parseZhantNewsList parse url:{} ,is null!!", url);
            return null;
        }
        return parseNewsListUS(doc, Constants.lang_zh_hant);
        // return parseNewsListHK(doc);
    }

    /*
     * 获取CNBC个股资讯首页对象并解析新闻条目列表
     */
    public List<YahooItem> parseNewsList(String url) {// url="http://finance.yahoo.com/quote/AAPL"
        Document doc = htmlPrase.parseDocumentFromUrl(url, true);
        if (doc == null) {
            logger.warn("SinaNewsListUitl parseNewsList parse url:{} ,is null!!", url);
            return null;
        }
        return parseNewsListUS(doc, Constants.lang_en);
    }

    private List<YahooItem> parseNewsListUS(Document doc, String lang) {
        List<YahooItem> result = new ArrayList<YahooItem>();

        Elements es = doc.select("script");
        for_:
        for (int i = 0; es != null && i < es.size(); i++) {
            if (es.get(i).childNodeSize() != 1)
                continue for_;
            // (function (root)
            String html = es.get(i).childNode(0).outerHtml();
            if (!StringUtils.startsWith(StringUtils.stripToEmpty(html), "(function (root)"))
                continue for_;
            html = StringUtils.substringAfter(html, "root.App.main =");
            html = StringUtils.substringBeforeLast(html, "(this)");
            html = StringUtils.substringBeforeLast(html, ";");
            JsonNode jd = UtilJson.readValue(html, JsonNode.class);
            // Optional.ofNullable(doc.select("meta[name=keywords]").first())
            // .map(me ->
            // StringUtils.strip(StringUtils.substringBefore(me.attr("content"),
            // ",")))//
            // .orElse(
            String keyWord = Optional.ofNullable(doc.select("title").first())//
                    .map(vs -> StringUtils.replace(vs.text(), "：", ":"))
                    .map(ti -> StringUtils.strip(StringUtils.substringBefore(ti, ":"))).orElse(null);
            Optional.ofNullable(jd.findValue("YFINANCE:" + keyWord + ".mega"))//
                    .map(vd -> vd.at("/data/stream_items"))//
                    .ifPresent(vds -> {
                       for_1: for (JsonNode jnd : vds) {
                            if ("article".equals(jnd.at("/type").asText())) {// 文章
                                YahooItem item = new YahooItem();
                                item.setTitle(jnd.at("/title").asText(null));
                                item.setLink(org.jsoup.helper.StringUtil.resolve(doc.baseUri(),
                                        jnd.at("/url").asText(null)));
                                if (StringUtils.isEmpty(item.getLink())) {// 补充链接
                                    item.setLink(org.jsoup.helper.StringUtil.resolve(doc.baseUri(),
                                            jnd.at("/link").asText(null)));
                                }

                                Optional.ofNullable(StringUtils.stripToNull(jnd.at("/images/original/url").asText()))
                                        .ifPresent(v00 -> {
                                            String mainPic = org.jsoup.helper.StringUtil.resolve(doc.baseUri(), v00);
                                            if (StringUtils.isNotBlank(mainPic))
                                                item.setMainPic(mainPic);
                                        });

                                item.setSource(jnd.at("/publisher").asText("yahoo.com"));
                                if (item.getSource().matches("(!?)Financial\\s+Times")) {
									continue for_1;
                                }

                                item.setNewsTime(new Date(jnd.at("/pubtime").asLong()));
                                item.setLang(lang);
                                if (StringUtils.isNotBlank(item.getLink()) //
                                        && item.getNewsTime() != null //
                                        && StringUtils.isNotBlank(item.getTitle())//
                                        && !StringUtils.containsIgnoreCase(item.getSource(), "CNBC")// CNBC重复了
                                        ) {
                                    result.add(item);
                                } else {
                                    logger.warn("yahoo新闻列表解析异常了baseUri:{},itme:{}", doc.baseUri(), item);
                                }
                            }
                        }
                    });
        }
        return result;
    }

    public static void main(String[] args) throws IOException {

        System.out.println("Financial Times".matches("(!?)Financial\\s+Times"));


        String url2 = "http://finance.yahoo.com/quote/AAPL";

        // Connection connection = Jsoup.connect(url2)
        // Connection connection = Jsoup.connect(urlHK)
        // url2 = "https://hk.finance.yahoo.com/q?s=AAPL";
        url2 = "https://finance.yahoo.com/q?s=AAPL";
        // url2 = "https://hk.finance.yahoo.com/q?s=0070.HK";
        Connection connection = Jsoup.connect(url2).userAgent(
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36")
                // .header("x-client-data",
                // "CIq2yQEIpbbJAQjEtskBCP2VygEIwpjKAQjwnMoB")
                .header("x-client-data", RandomStringUtils.randomAlphanumeric(40));
        connection.proxy("127.0.0.1", 1080);
        org.jsoup.nodes.Document doc = connection.timeout(10000).get();

        List<YahooItem> items = new YahooNewsListUitl().parseNewsListUS(doc, Constants.lang_en);
        // List<YahooItem> items = new YahooNewsListUitl().parseNewsListHK(doc);
        System.out.println(items);
    }

    public static class YahooItem {
        private Date newsTime;

        private String title;
        private String link;
        private String source;
        private String lang;
        private String mainPic;

        public String getMainPic() {
            return mainPic;
        }

        public void setMainPic(String mainPic) {
            this.mainPic = mainPic;
        }

        public String getLang() {
            return lang;
        }

        public void setLang(String lang) {
            this.lang = lang;
        }

        public Date getNewsTime() {
            return newsTime;
        }

        public void setNewsTime(Date newsTime) {
            this.newsTime = newsTime;
        }

        public String getTitle() {
            return title;
        }

        public void setTitle(String title) {
            this.title = title;
        }

        public String getLink() {
            return link;
        }

        public void setLink(String link) {
            this.link = link;
        }

        public String getSource() {
            return source;
        }

        public void setSource(String source) {
            this.source = source;
        }

        @Override
        public String toString() {
            return "YahooItem [newsTime=" + newsTime + ", source=" + source + ", title=" + title + ", link=" + link
                    + "]";
        }

    }

}
